# Copyright (C) 2001, 2003-2004, 2006, 2008-2010 The Written Word, Inc.
# Copyright (C) 2000-2001 The Written Word, LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id$

import atexit
import os
import string
import sys
import types

from urlparse import urlparse
from UserString import UserString

import depot
import depotdb
import misclib
import myio
import mysig

from classutil import AbstractMeta, AbstractProperty, abstract
from misclib import VERBOSE_DEPOT, VERBOSE_LOCAL_DEPOT, urljoin
from packagenames import PackageName, PackageNames
from xmlutil import XMLError, XMLSyntaxError

# options for depots.set()
SEARCH_ACCEPT_NO_DEPOTS = 1	# caller doesn't require a depot

# number of bytes in a checksum file
CHECKSUM_FILESIZE = 33          # pycurl can't determine this for all protocols

# IO Handler
io = myio.MyIO ()

# exceptions
class DepotError (Exception):
  """Base class for all exceptions raised by this module"""

  def __init__ (self, msg = None):
    self.msg = msg

  def __str__ (self):
    return self.msg

class DepotAccessDeniedError (DepotError):
  """access denied (authorization insufficient)"""

class DepotAccessError (DepotError):
  """unable to access depot"""

class DepotArchiveInitError (DepotError):
  """internal archive initialization failed"""

class DepotFileNotFoundError (DepotError):
  """file not found in depot"""

  def __init__ (self, file = None):
    self.file = file

  def __str__ (self):
    return self.file

class DepotGPGError (DepotError):
  """GPG signature for depot-db.xml failed"""

class DepotNoLocalDepot (DepotError):
  """no local depot specified with --local-depot"""

class DepotNoValidDepots (DepotError):
  """no valid depots"""

class DepotNotFoundError (DepotError):
  """depot not found"""

class DepotPackageVerNotFoundError (DepotError):
  """requested version of this package not found in depot"""

class DepotPermissionDeniedError (DepotError):
  """permission denied"""

class DepotProtocolUnsupportedError (DepotError):
  """protocol unsupported"""

class DepotProxyAuthenticationRequiredError (DepotError):
  """proxy authentication required"""

class DepotSftpAuthenticationError (DepotError):
  """authentication failure"""

  def __init__ (self, host):
    self.host = host

  def __str__ (self):
    return self.host

class DepotSftpKnownHostsFileNotFoundError (DepotError):
  """ssh_known_hosts file not found"""

class DepotSftpKnownHostsFileNotSpecifiedError (DepotError):
  """ssh_known_hosts file not specified"""

class DepotSftpKnownHostsPathNotFoundError (DepotError):
  """path specified for ssh_known_hosts not found"""

class DepotSftpInvalidKnownHostError (DepotError):
  """host unknown in ssh_known_hosts"""

class DepotSftpPrivateKeyEncryptedError (DepotError):
  """private key encrypted with passphrase"""

class DepotSftpPrivateKeyFileNotFoundError (DepotError):
  """private key file not found"""

class DepotSftpPrivateKeyFileNotSpecifiedError (DepotError):
  """private key file not specified"""

class DepotSSLError (DepotError):
  """SSL error"""

  def __init__ (self, msg):
    self.error = msg

  def __str__ (self, msg):
    return self.error

class DepotTempdirCreationError (DepotError):
  """error creating temporary directory for any-url package retrieval"""


class ProgressAdaptor (object):
  """Adaptor to allow use of `TextProgressMeter` with pycurl.

  pycurls PROGRESSFUNCTION repeatedly calls the named function between
  downloaded chunks, but pkgutils potentially wants to show progress
  of several related files along a single `TextProgressMeter`. A
  `ProgressAdaptor` maintains the book-keeping to make that possible,
  while providing a `progress_cb` method to pass as the PROGRESSFUNCTION.
  """

  def __init__(self, progressmeter, fetched_bytes=0, additional_bytes=0):
    """Return an adaptor wrapped around `progressmeter`.

    :Parameters:
        `progressmeter`: TextProgressMeter
            Re-use `progressmeter` for fetching related files.
        `fetched_bytes`: integer
            Number of bytes already fetched as previous files using
            this `progressmeter`.
        `additional_bytes`: integer
            Additional bytes yet to be downloaded with further files
            using this `progressmeter`.
    """

    self.progressmeter = progressmeter
    self.fetched_bytes = fetched_bytes
    self.additional_bytes = additional_bytes

    self.running_total = 0

  def progress_cb (self, dltotal, dlnow, ultotal, ulnow):
    """Callback function for updating `progressmeter`.

    :Parameters:
        `dltotal`: float
            Total number of bytes to download for the current file.
        `dlnow`: float
            Number of bytes downloaded so far for the current file.
        `ultotal`: not used
        `ulnow`: not used

    Set pycurl.PROGRESSFUNCTION to call this function to redraw
    `progressmeter` taking into account other related files already
    fetched, and not yet started.
    """

    # pycurl tells us the total number of bytes that need to be fetched
    # for just the current file in `dltotal`, but when we call the start
    # method for `progressmeter` before any bytes have been fetched for
    # any files in the current group, we need to tell it the aggregated
    # size for the entire group:
    if dlnow == 0 or self.fetched_bytes == 0 and self.running_total == 0:
      self.progressmeter.start (dltotal + self.additional_bytes)

    self.running_total = int (dltotal) + self.fetched_bytes
    self.progressmeter.update (int (dlnow) + self.fetched_bytes)

  def get_running_total (self):
    """Return the number of bytes downloaded so far for the whole group."""
    return self.running_total


# proxy authentication variables
class ProxyAuth:
  def __init__ (self):
    self.host = ''
    self.login = ''
    self.passwd = ''

class ProtocolAuth:
  def __init__ (self):
    self.login = ''
    self.passwd = ''

    self.sftp_known_hosts_path = None	# path to ssh_known_hosts
    self.sftp_private_key_path = None	# path to ssh private key

# version number and revision class
class DepotBase (UserString):
  """Version number and revision class.

  New style class method lookup resolution only finds the "stringy"
  methods if we inherit a "stringy" superclass.  Unfortunately, inheriting
  the builtin "str" makes our type into "str" - foiling the type switched
  code looking for "InstanceType"."""
  
  def __init__ (self, dep):
    if type (dep) == types.InstanceType:
      url = dep.url
    else:
      url = dep

    # This is better, but doesn't work for classic classes:
    #    super (Depot, self).__init__ (dep)
    # So, we're forced to hardcode the superclass for now:
    UserString.__init__ (self, url)
    
    self.archive = 0			# if depot is a zip archive

    self.url = url			# parent url
    self.pkg_url = url			# where to find packages
    self.type = urlparse (url).scheme   # URL scheme ('http', ...)
    self.local_path = None

    self.proxy = ProxyAuth ()
    self.auth = ProtocolAuth ()

    self.dist = None			# distribution name
    self.packages = []			# packages contained in depot

    self.indent_progressmeter = 4
    self.verify_checksum = False

  # output depot path
  def __str__ (self):
    return self.url

  def set_proxy (self):
    pass

  def set_auth (self):
    pass

  def fetch (self, pkg):
    pass

  # if package exists in local depot
  def _db_exists (self, base_name, suffix = ''):
    if not os.path.exists (os.path.join (self.local_path,
                                         base_name + suffix)):
      return False
    return True

  def open (self, path, mode):
    if self.local_path and os.path.exists (self.local_path + '/' + path):
      return open (self.local_path + '/' + path, 'r')
    else:
      return (self.urlopen (urljoin (self.pkg_url, path)))

# list of repositories
class DepotsBase (object):
  # Raise an exception at class instantiation time if subclasses
  # contain any (unoverridden) AbstractProperty's or @abstract methods:
  __metaclass__ = AbstractMeta

  # Set these appropriately in subclass
  XML_FILE = AbstractProperty ('XML_FILE')
  DB_SUFFIX = AbstractProperty ('DB_SUFFIX')

  def __init__ (self, add_package, assign_depot_vars,
                local_depot, msg_callback, package_names,
                tmpbasedir, var, invalid_package_callback = None):
    self.add_package = add_package
    self.assign_depot_vars = assign_depot_vars
    self.local_depot = local_depot
    self.msg_callback = msg_callback
    self.package_names = package_names
    self.tmpbasedir = tmpbasedir
    self.var = var
    self.invalid_package_callback = invalid_package_callback

    self.depots = []
    self.temp_package_dir = None		# where to store
                                 		# any-url packages

    self.local_depot_is_temporary = False	# if local path is a
						# temporary path because
						# a permanent path was
						# not specified

  @abstract
  def add_packages (self, packages, archive):
    """Apply the add_package callback.
    
    @param packages: As returned by `read_db_xml`
    @param archive: As returned by `archive_init`
    """
    pass
  
  @abstract
  def archive_init (self, path):
    """Initialise an archive object in the concrete class

    @param path: File path to the archive"""
    pass

  @abstract
  def depot_init (self, dep):
    """Initialise a depot object in the concrete class

    @param dep: The depot object
    """
    pass
  
  @abstract
  def read_db_xml (self, archive, dep, basename):
    """Read an XML file

    @param archive: As returned by `archive_init`
    @param dep: A depot object
    @param basename: The name of the file to find and read

    @returns A parsed XML DB entry"""
    pass

  def sub_depot_filter (self, sub_depot):
    """Return `True` to filter out (ignore) this sub_depot

    For each depot discovered, this filter is called for any sub-depots
    it may contain.  They are only added to the list of depots to be
    searched when `False` is returned.

    @param sub_depot: A sub-depot of one of the discovered depots

    @returns `True` to ignore, or `False` to search
    """
    return False

  def validate_xml_callback (self, basename, dep):
    """Use this callback for performing additional validation on `basename`"""
    pass

  def verify_depots_callback (self, depots, options):
    """Use this callback for verifying the validity of `depots`

    @returns a new (verified) `depots` to replace the original
    """
    return depots

  def xml_file_filter (self, valid_depots, dep, found_depot_db_xml):
    """I have no idea how to characterize this method

    This is the chunk of code that differed between the older revisions of
    {pb,pkg,sb}utils-x.x.x/lib/depot/__init__.py.  Now the common code is
    in a shared lib/baseclase/depotbase.py, and this method is called in
    just the right spot, with just the right parts factored out into
    the new subclass override for this method in each instance that
    everything continues to work just like it always did.

    @param valid_depots: the current list of validated depots
    @param dep: the current depot
    @param found_depot_db_xml: True if we've found `depot-db.xml` 

    @returns `True` to cause the calling loop to continue
    """
    return False

  # remove temporary package directory
  def remove_temp_package_dir (self, msg_callback = None):
    if self.temp_package_dir:
      if msg_callback:
        msg_callback ('begin', 'removing temporary package directory', 2)
      misclib.rmdir (self.temp_package_dir)
      if msg_callback:
        msg_callback ('end', 'done', 2)

      mysig.unregister (self.remove_temp_package_dir)
      self.temp_package_dir = None

  # remove local repository
  def remove_local_depot (self, local_depot, msg_callback = None):
    if not local_depot and self.local_depot:
      if msg_callback:
        msg_callback ('begin', 'removing temporary \
local depot directory', 2)
      misclib.rmdir (self.local_depot)
      if msg_callback:
        msg_callback ('end', 'done', 2)

      mysig.unregister (self.remove_local_depot)

  def set (self, depot_names, options = []):
    valid_depots = []			# depots we can use
    search_depots = False
    search_package_names = []		# packages to search for

    # make copy as invalid_package_callback might modify in place
    for package_name in self.package_names[:]:
      if not package_name.isfile ():
        # Unless we already have a local file, try to fetch the file from
        # (remote) url into a temporary local depot for processing later on.
        try:
          self.temp_package_dir = misclib.mkstempdir ()
        except misclib.MisclibTempDirCreationError, msg:
          raise DepotTempdirCreationError (src (msg))

        mysig.register (self.remove_temp_package_dir)

        dep = init (None, self.temp_package_dir,
                    package_name.remote_depot ())
        dep.indent_progressmeter = 2
        dep.verify_checksum = False
        self.depot_init (dep)

        if dep.exists (package_name.remote_path ()):
          # mysig.register() call above avoids a race if the program
          # exits with a signal, and the atexit.register() call below
          # ensures we also clean up when the program exits normally.
          atexit.register (self.remove_temp_package_dir, self.msg_callback)

          # To minimize chatter, only admit that we created a temporary
          # directory if we actually need it...
          self.msg_callback ('begin', 'creating temporary \
directory', 0, VERBOSE_LOCAL_DEPOT)
          self.msg_callback ('end', self.temp_package_dir,
                                     0, VERBOSE_LOCAL_DEPOT)
          io.stdout (0, '', 'retrieving ' +
                     package_name.geturl () + ' ...\n')
        elif package_name.getscheme () != 'file':
          raise DepotFileNotFoundError ('%s: File not found.' %
                                        package_name.geturl ())
        else:
          # ...otherwise, quietly remove it and pretend it was never
          # here (in which case we won't be needing the atexit handler
          # either):
          self.remove_temp_package_dir ()

        self.assign_depot_vars (dep)

        # If we successfully fetched a local copy of the remote file,
        # replace the remote package_name object with a file:// scheme
        # package_name:
        pkgfile = package_name.basename ()
        if dep.exists (pkgfile):
          dep.get (pkgfile, show_progress = True)
          package_name = \
            PackageName (os.path.join (self.temp_package_dir,
                                       pkgfile))

      if package_name.isfile ():
        path = package_name.normpath ()

        dep = init (None, self.local_depot, os.path.dirname (path))

        self.assign_depot_vars (dep)

        self.msg_callback ('begin', 'reading ' +
                urljoin (dep.pkg_url, os.path.basename (path)),
                0, VERBOSE_DEPOT)

        try:
          archive = self.archive_init (path)
          if archive:
            archive.path = path

            # set depot path to the archive file
            dep.local_path = path
            dep.url = package_name.getscheme () + "://" + path
            dep.archive = archive

            if archive.exists (self.XML_FILE):
              db_entry = self.read_db_xml (archive, dep, self.XML_FILE)
            elif archive.exists (os.path.basename (path)):
              db_entry = self.read_db_xml (archive, dep,
                                           os.path.basename (path))
            else:
              if self.invalid_package_callback:
                self.invalid_package_callback (package_name.name)

              self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
              self.msg_callback ('error', 'an input archive \
was specified but did not contain either a "%s" or "%s".  Archives \
must contain one of these files. skipping.' % (self.XML_FILE,
                                                 os.path.basename (path)))
              continue
          else:
            db_entry = self.read_db_xml (dep, dep, path)

          if not db_entry:
            self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
            self.msg_callback ('error', 'unable to parse the \
XML file. please check "%s". it does not appear to follow \
%s(4). skipping.' % (path, self.XML_FILE))
            continue

        except DepotArchiveInitError, msg:
          if self.invalid_package_callback:
            self.invalid_package_callback (package_name.name)

          self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
          self.msg_callback ('error', str (msg))
          continue
        except XMLError, msg:
          self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
          self.msg_callback ('error', str (msg))
          continue

        self.depots.append (dep)
        valid_depots.append (dep)

        self.msg_callback ('end', 'done', 0, VERBOSE_DEPOT)

        self.add_packages (db_entry, archive)
      else:
        self.add_package (package_name.name)
        search_package_names.append (package_name.name)
        search_depots = True		# search through list of
					# depots for package
    else:
      # If packages specified, and at least one valid package
      # name remains, search repositories looking for it
      if self.package_names and search_package_names:
        search_depot = True

      # Also search repositories if no packages given
      if not self.package_names:
        search_depots = True

    # initialize depots
    if search_depots and depot_names:
      for depot_name in depot_names:
        # create temp directory for local depot
        if is_remote (depot_name) and not self.local_depot:
          self.msg_callback ('begin', 'creating temporary \
local depot directory', 0, VERBOSE_LOCAL_DEPOT)

          try:
            self.local_depot = misclib.mkstempdir (self.tmpbasedir)
            self.local_depot_is_temporary = True
          except misclib.MisclibTempdirCreationError, msg:
            raise DepotNoLocalDepot (str (msg))

          mysig.register (self.remove_local_depot, None)
          self.msg_callback ('end', self.local_depot, 0, VERBOSE_LOCAL_DEPOT)

        try:
          dep = init (self.var, self.local_depot, depot_name)
        except DepotProtocolUnsupportedError, msg:
          self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
          self.msg_callback ('error', str (msg) + ' removing ' +
                             depot_name + ' from list of depots.')
          continue

        if dep.is_remote () and not self.local_depot:
          raise DepotNoLocalDepot ('remote depots were specified but a \
--local-depot option was not given. please rerun specifying a local \
path where packages from remote repositories can be copied to.')

        try:
          self.assign_depot_vars (dep)
        except (DepotSftpKnownHostsFileNotFoundError,
                DepotSftpKnownHostsFileNotSpecifiedError,
                DepotSftpPrivateKeyFileNotFoundError,
                DepotSftpPrivateKeyFileNotSpecifiedError), msg:
          self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
          self.msg_callback ('error', str (msg))
          continue

        self.depots.append (dep)

      self.msg_callback ('end', 'reading depot databases', 0,
                         VERBOSE_DEPOT)

      #??? self.depots = self.verify_depots_callback (self.depots, options)

      for dep in self.depots:
        depot_db_xml = False		# if depot-db.xml exists
        db_xml = False			# if xx-db.xml exists

        if dep.archive:			# skip archives
          continue

        # if parent depot contains more depots, add to list
        self.msg_callback ('begin', urljoin (dep.pkg_url, 'depot-db.xml'),
                           2, VERBOSE_DEPOT)

        try:
          if dep.exists ('depot-db.xml'):
            depot_db_xml = True

            dep.fetch ('depot-db.xml')	# make copy in local depot

            self.msg_callback ('end', 'done', 2, VERBOSE_DEPOT)

            self.validate_xml_callback ('depot-db.xml', dep)

            sub_depots = depotdb.read_depotdb_xml ('depot-db.xml', dep)

            # create new depots
            for sub_depot in sub_depots:
              if self.sub_depot_filter (sub_depot, search_package_names):
                continue

              # make sure distribution matches one we are interested in
              if sub_depot.dist_ver:
                if sub_depot.dist_ver not in dep.dist_ver:
                  continue

              self.msg_callback ('end', 'adding ' +
                      urljoin (dep.pkg_url, sub_depot.name),
                                 4, VERBOSE_DEPOT)

              if sub_depot.name:
                new_dep = init (self.var, self.local_depot, dep,
                                sub_depot.name)
                self.assign_depot_vars (new_dep)

                # record update type
                if sub_depot.category:
                  new_dep.update_type = sub_depot.category
                elif hasattr (dep, 'update_type'):
                  new_dep.update_type = dep.update_type

                if (sub_depot.package and
                    not sub_depot.package in new_dep.packages):
                  new_dep.packages.append (sub_depot.package)

                self.depots.append (new_dep)
              else:
                dep.packages.append (sub_depot.package)
          else:
            self.msg_callback ('end', 'failed (not found)', 2,
                               VERBOSE_DEPOT)
        except DepotAccessDeniedError:
          self.msg_callback ('end', 'failed (access denied)', 2,
                             VERBOSE_DEPOT)
          continue
        except DepotPermissionDeniedError:
          self.msg_callback ('end', 'failed (permission denied)',
                             2, VERBOSE_DEPOT)
          continue
        except DepotProxyAuthenticationRequiredError:
          self.msg_callback ('error', 'access denied to proxy \
server (' + dep.proxy.host + '). either the proxy login id or \
password was incorrect or not specified. removing ' + dep.url +
                             ' from list of depots.')
          continue
        except DepotSftpAuthenticationError, msg:
          self.msg_callback ('error', 'unable to authenticate to ' +
                                     msg.host + ' using the private key \
specified. please either specify the correct private key or ensure \
the remote host has the contents of "' + dep.auth.sftp_private_key_path +
                                     '.pub" in ~' + dep.auth.login +
                                     '/.ssh/authorized_keys. removing ' +
                             dep.url + ' from list of depots.')
          continue
        except (DepotSftpInvalidKnownHostError,
                DepotSftpPrivateKeyEncryptedError,
                XMLError), msg:
          self.msg_callback ('error', str (msg) + ' removing ' +
                             dep.url + ' from list of depots.')
          continue

        if self.xml_file_filter (valid_depots, dep, depot_db_xml):
          continue

      self.depots = valid_depots

    # separate depot output and package installation
    if True or self.depots: #???
      self.msg_callback ('end', '', 0, VERBOSE_DEPOT)

    # must have at least one valid depot
    if not self.depots:
      if depot_names:
        if self.package_names and not search_package_names:
          raise DepotNoValidDepots ('none of the package names specified \
on the command-line were valid packages.')
        else:
          raise DepotNoValidDepots ('while some depot paths were given (' + \
          string.join (depot_names, ',') + '), none specified a path we \
could use. The repository must contain either a "depot-db.xml" file \
specifying further subdirectories to walk, a "%s" file specifying a \
package definition, or "[package]%s" files specifying package \
definitions for the input package names.' % (self.XML_FILE, self.DB_SUFFIX))
      else:
        if not SEARCH_ACCEPT_NO_DEPOTS in options:
          raise DepotNoValidDepots ('no depots given. please specify \
at least one depot with the --depot option or the path of a \
%s(4) file or a %s archive file.' % (self.XML_FILE, self.DB_SUFFIX))

  def append (self, dep):
    self.depots.append (dep)

  def __getitem__ (self, key):
    return self.depots[key]

  def __iter__ (self):
    return self.depots.__iter__ ()

  def __len__ (self):
    return len (self.depots)


# test if URL is remote
def is_remote (url):
  if url.startswith ('ftp://'):
    return True
  elif url.startswith ('sftp://'):
    return True
  elif url.startswith ('http://'):
    return True
  elif url.startswith ('https://'):
    return True
  else:
    return False

def init (var, local_depot, dep, sub_depot = None):
  # dep is either a URL or an instance of the Depot class. if the
  # latter, dep is the parent URL of sub_depot.
  if type (dep) == types.InstanceType:
    url = dep.url
  else:
    url = dep

  if url.startswith ('ftp://'):
    import depot.ftp
    return (depot.ftp.ftp (var, local_depot, dep, sub_depot))
  elif url.startswith ('sftp://'):
    try:
      import depot.sftp
      return (depot.sftp.sftp (var, local_depot, dep, sub_depot))
    except ImportError:
      raise DepotProtocolUnsupportedError ('sftp protocol not \
supported on this platform. one or more of the dependencies \
required to support it are unavailable.')
  elif url.startswith ('http://'):
    import depot.http
    return (depot.http.http (var, local_depot, dep, sub_depot))
  elif url.startswith ('https://'):
    import depot.https
    return (depot.https.https (var, local_depot, dep, sub_depot))
  elif url.startswith ('file://'):
    import depot.file
    return (depot.file.file (var, dep, sub_depot))
  else:
    import depot.file
    return (depot.file.file (var, 'file://' + dep, sub_depot))
